// --------------------------------------------------------------------------------------------------------------------
// <copyright file="MetricsAutomation.cs" company="Microsoft Corporation">
// The MIT License (MIT)
// 
// Copyright (c) 2014, Microsoft Corporation
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in
//  all copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
//  THE SOFTWARE.
// </copyright>
// --------------------------------------------------------------------------------------------------------------------
namespace Test.Robotics.Navigation
{
    using System;
    using System.Collections.Generic;
    using System.Diagnostics;
    using System.IO;
    using System.Threading;
    using Microsoft.Robotics.IO;
    using Microsoft.Robotics.Numerics;
    using Microsoft.Robotics.Runtime;
    using Microsoft.Robotics.Tracing;
    using Microsoft.VisualStudio.TestTools.UnitTesting;
    using Test.Robotics.Metrics;    
    using Test.Robotics.Navigation.Metrics.PathPlanningAndTracking;

    /// <summary>
    /// Hosts all the navigation metrics test cases, one per dataset/configuration
    /// </summary>
    [TestClass]
    [DeploymentItem(@"Microsoft.Robotics.Vision.Native.dll")]
    public class MetricsAutomation
    {
        /// <summary>
        /// Configuration ID for Robot 1 on 2nd Floor Building 115
        /// </summary>
        private const int Bldg115Floor2Robot1ConfigId = 29;

        /// <summary>
        /// Configuration ID for Robot 2 on 2nd Floor Building 115
        /// </summary>
        private const int Bldg115Floor2Robot2ConfigId = 78;

        /// <summary>
        /// Configuration ID for Robot 1 on 4th Floor Building 115
        /// </summary>
        private const int Bldg115Floor4Robot1ConfigId = 86;
        
        /// <summary>
        /// A starting pose for Robot 1 
        /// </summary>
        private readonly Pose2D startPoseBldg115Floor2Robot1 = new Pose2D(217, 538, 0);

        /// <summary>
        /// A starting pose for Robot 2
        /// </summary>
        private readonly Pose2D startPoseBldg115Floor2Robot2 = new Pose2D(217, 548, 0);

        /// <summary>
        /// A starting pose for Robot 1 
        /// </summary>
        private readonly Pose2D startPoseBldg115Floor4Robot1 = new Pose2D(2219, 1290, 4.712);

        /// <summary>
        /// Initialize this test class
        /// </summary>
        /// <param name="tc">The test context</param>
        [ClassInitialize]
        public static void ClassInitialize(TestContext tc)
        {
            Trace.Listeners.Add(new TextWriterTraceListener(@"C:\temp\NavMetricsAutomation.log", "myListener"));
            Trace.TraceInformation("Test message.");
            //// You must close or flush the trace to empty the output buffer.
            //// Trace.Flush();
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A050.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopEV4A050()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop", "EV4A050", new Pose2D(200, 200, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A050 with turret panning at 5 degree increments, pausing for 200 milliseconds at each pan angle.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopTurretEV4A050()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop with turret", "EV4A050", new Pose2D(200, 200, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A050 with turret panning at 30 degree increments, pausing for 500 milliseconds at each pan angle.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopTurretFastScanEV4A050()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop with turret fast scan", "EV4A-050", new Pose2D(600, 600, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A-026.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopEV4A026()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop", "EV4A-026", new Pose2D(200, 200, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A-026 with turret panning at 5 degree increments, pausing for 200 milliseconds at each pan angle.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopTurretEV4A026()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop with turret", "EV4A-026", new Pose2D(200, 200, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A-026 with turret panning at 30 degree increments, pausing for 500 milliseconds at each pan angle.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopTurretFastScanEV4A026()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop with turret fast scan", "EV4A-026", new Pose2D(600, 600, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A-036.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopEV4A036()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop", "EV4A-036", new Pose2D(200, 200, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A-036 with turret panning at 5 degree increments, pausing for 200 milliseconds at each pan angle.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopTurretEV4A036()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop with turret", "EV4A-036", new Pose2D(200, 200, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 small loop recorded with EV4A-036 with turret panning at 30 degree increments, pausing for 500 milliseconds at each pan angle.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CarpetLoopTurretFastScanEV4A036()
        {
            // the name passed in must match the name of the row in SQL            
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Carpet loop with turret fast scan", "EV4A-036", new Pose2D(600, 600, 0), 800, 800);
        }

        /// <summary>
        /// Runs the 115/2 cement loop recorded with EV4A-050.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CementLoopEV4A050()
        {
            // the name passed in must match the name of the row in SQL
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Cement loop with turret fast scan", "EV4A-050", new Pose2D(200, 2000, 0), 3500, 3500);
        }

        /// <summary>
        /// Runs the 115/2 cement loop recorded with EV4A-036.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CementLoopEV4A036()
        {
            // the name passed in must match the name of the row in SQL
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Cement loop with turret fast scan", "EV4A-036", new Pose2D(200, 2000, 0), 3500, 3500);
        }

        /// <summary>
        /// Runs the 115/2 cement loop recorded with EV4A-026.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CementLoopEV4A026()
        {
            // the name passed in must match the name of the row in SQL
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Cement loop with turret fast scan", "EV4A-026", new Pose2D(200, 2000, 0), 3500, 3500);
        }

        /// <summary>
        /// Runs the 115/2 cement loop recorded in reverse direction with EV4A-050.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CementLoopReverseEV4A050()
        {
            // the name passed in must match the name of the row in SQL
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Cement loop with turret fast scan reverse", "EV4A-050", new Pose2D(3500 - 200, 3500 - 2000, 0), 3500, 3500);
        }

        /// <summary>
        /// Runs the 115/2 cement loop recorded  in reverse direction with EV4A-036.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CementLoopReverseEV4A036()
        {
            // the name passed in must match the name of the row in SQL
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Cement loop with turret fast scan reverse", "EV4A-036", new Pose2D(3500 - 200, 3500 - 2000, 0), 3500, 3500);
        }

        /// <summary>
        /// Runs the 115/2 cement loop recorded in reverse direction with EV4A-026.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_CementLoopReverseEV4A026()
        {
            // the name passed in must match the name of the row in SQL
            this.RunOfflineSLAMWithTurretMetricsManifest("B115/2 Cement loop with turret fast scan reverse", "EV4A-026", new Pose2D(3500 - 200, 3500 - 2000, 0), 3500, 3500);
        }    

        /// <summary>
        /// Runs the ACES external data set
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(TestTimeout.Infinite)]
        [TestCategory("Metrics")]
        public void SLAMMetrics_External_ACES()
        {
            // the name passed in must match the name of the row in SQL
            this.RunOfflineExternalDatasetSLAMMetricsManifest("ACES Bldg Austin", new Pose2D(600, 900, 0), 900, 900);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 1).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics1()
        {
            this.RunOnlineNavMetricsManifest(15, 1, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }
        
        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 2).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics2()
        {
            this.RunOnlineNavMetricsManifest(15, 2, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 3).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics3()
        {
            this.RunOnlineNavMetricsManifest(15, 3, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 4).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics4()
        {
            this.RunOnlineNavMetricsManifest(15, 4, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 5).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics5()
        {
            this.RunOnlineNavMetricsManifest(15, 5, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 6).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics6()
        {
            this.RunOnlineNavMetricsManifest(15, 6, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 7).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics7()
        {
            this.RunOnlineNavMetricsManifest(15, 7, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 8).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics8()
        {
            this.RunOnlineNavMetricsManifest(15, 8, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 9).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics9()
        {
            this.RunOnlineNavMetricsManifest(15, 9, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 1 with seed of 10).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics10()
        {
            this.RunOnlineNavMetricsManifest(15, 10, Bldg115Floor2Robot1ConfigId, this.startPoseBldg115Floor2Robot1, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 1).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics1Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 1, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 2).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics2Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 2, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 3).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics3Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 3, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 4).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics4Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 4, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 5).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics5Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 5, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 6).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics6Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 6, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 7).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics7Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 7, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 8).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics8Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 8, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 9).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics9Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 9, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 10).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void CementLoopMetrics10Robot2()
        {
            this.RunOnlineNavMetricsManifest(15, 10, Bldg115Floor2Robot2ConfigId, this.startPoseBldg115Floor2Robot2, PathPlanningAndTrackingManifest.GenerateCementLoopLandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 1).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics1()
        {
            this.RunOnlineNavMetricsManifest(15, 1, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 2).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics2()
        {
            this.RunOnlineNavMetricsManifest(15, 2, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 3).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics3()
        {
            this.RunOnlineNavMetricsManifest(15, 3, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 4).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics4()
        {
            this.RunOnlineNavMetricsManifest(15, 4, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 5).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics5()
        {
            this.RunOnlineNavMetricsManifest(15, 5, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 6).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics6()
        {
            this.RunOnlineNavMetricsManifest(15, 6, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 7).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics7()
        {
            this.RunOnlineNavMetricsManifest(15, 7, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 8).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics8()
        {
            this.RunOnlineNavMetricsManifest(15, 8, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 9).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics9()
        {
            this.RunOnlineNavMetricsManifest(15, 9, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Runs the "cement loop" metrics (robot 2 with seed of 10).
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [Timeout(70 * 60 * 1000)] // 70 minutes
        [TestCategory("Metrics")]
        public void FouthFloorMetrics10()
        {
            this.RunOnlineNavMetricsManifest(15, 10, Bldg115Floor4Robot1ConfigId, this.startPoseBldg115Floor4Robot1, PathPlanningAndTrackingManifest.GenerateBldg115Floor4LandmarkIds);
        }

        /// <summary>
        /// Processes all the navigation data directories in the recordings folder referred to in the database under the 
        /// configuration with name "B115/2 CementLoop Pioneer @ 20cm threshold".
        /// This only needs to be run when new data is added to that folder.  Any old (already processed) data should 
        /// be moved before running this or it will be duplicated in the database.
        /// </summary>
        [Priority(0)]
        [TestMethod]
        [TestCategory("Metrics")]
        public void ProcessPioneerNavData20cmThreshold()
        {
            this.RunProcessPioneerNavMetricsManifest("B115/2 CementLoop Pioneer @ 20cm threshold");
        }

        /// <summary>
        /// Configures and starts the offline SLAM metrics manifest.
        /// </summary>
        /// <param name="configurationName">The name of the configuration to load</param>
        /// <param name="robotName">The name of the robot that was used to record the data</param>
        /// <param name="startPose">Start pose on the map</param>
        /// <param name="mapHeight">Map height</param>
        /// <param name="mapWidth">Map width</param>
        private void RunOfflineSLAMWithTurretMetricsManifest(string configurationName, string robotName, Pose2D startPose, int mapHeight, int mapWidth)
        {
            TraceOut.Info(
                TraceContexts.Test, 
                "Starting Offline Slam for config {0}, robot {1}, with start pose {2}, map height {3}, map width {4}",
                configurationName,
                robotName,
                startPose,
                mapHeight,
                mapWidth);

            IEnumerable<Configuration> configSet = MetricsTestRunner.GetConfigurations(configurationName, robotName);
            foreach (Configuration config in configSet)
            {
                TraceOut.Info(
                    TraceContexts.Test,
                 "Starting configuration with recording {0}, ground truth {1}, iteration count {2}, configuration data {3} ",
                 config.Recording,
                 config.GroundTruthSource,
                 config.IterationCount,
                     config.ConfigurationData);

                Manifest m = NavigationManifest.CreateSlamMetricsManifest(config.Recording, config.GroundTruthSource, config.IterationCount, startPose, mapHeight, mapWidth, config.ConfigurationData, false);                
                MetricsTestRunner.Run(m, config, TimeSpan.FromHours(6), true);
            }
        }

        /// <summary>
        /// Configures and starts the offline SLAM metrics manifest.
        /// </summary>
        /// <param name="configurationName">The name of the configuration to load</param>
        /// <param name="startPose">Start pose on the map</param>
        /// <param name="mapHeight">Map height</param>
        /// <param name="mapWidth">Map width</param>
        private void RunOfflineExternalDatasetSLAMMetricsManifest(string configurationName, Pose2D startPose, int mapHeight, int mapWidth)
        {
            TraceOut.Info(
              TraceContexts.Test,
              "Starting Offline Slam for config {0}, with start pose {1}, map height {2}, map width {3}",
              configurationName,              
              startPose,
              mapHeight,
              mapWidth);

            Configuration config = MetricsTestRunner.GetConfigurationByName(configurationName);

            TraceOut.Info(
                TraceContexts.Test,
                "Starting configuration with recording {0}, ground truth {1}, iteration count {2}, configuration data {3} ",
                config.Recording,
                config.GroundTruthSource,
                config.IterationCount,
                config.ConfigurationData);

            Manifest m = NavigationManifest.CreateExternalDatasetSLAMMetricsManifest(config.Recording, config.GroundTruthSource, config.IterationCount, startPose, mapHeight, mapWidth, config.ConfigurationData, false);
            MetricsTestRunner.Run(m, config, TimeSpan.FromHours(6), true);
        }

        /// <summary>
        /// Configures and starts the Path Planning and Tracking metrics manifest.
        /// </summary>
        /// <param name="count">Number of waypoints to generate.</param>
        /// <param name="seed">Initial random seed.</param>
        /// <param name="configurationId">Database id of the configuration to use</param>
        /// <param name="startPose">The starting pose of the robot</param>
        /// <param name="landmarkIdFn">Function to generate the waypoints to traverse (taking count and seed).</param>
        private void RunOnlineNavMetricsManifest(int count, int seed, int configurationId, Pose2D startPose, Func<int, int, int[]> landmarkIdFn)
        {
            Configuration config = MetricsTestRunner.GetConfigurationById(configurationId); // must match the name of the row in SQL

            Manifest m = PathPlanningAndTrackingManifest.CreateCementLoopMetricsManifest(
                landmarkIdFn(count, seed),
                config.EnvironmentConfiguration,
                config.IterationCount,
                startPose,
                config.Robot,
                config.ConfigurationData,
                true);
            MetricsTestRunner.Run(m, config, TimeSpan.FromHours(20), false);

            Thread.Sleep(10000); // allow time for agent pipeline to shut down
        }

        /// <summary>
        /// Configures and starts the Pioneer Navigation Metrics Data Processor.
        /// </summary>
        /// <param name="configurationName">The name of the configuration to load</param>
        private void RunProcessPioneerNavMetricsManifest(string configurationName)
        {
            Configuration config = MetricsTestRunner.GetConfigurationByName(configurationName);
            foreach (string recordingDir in Directory.EnumerateDirectories(config.Recording))
            {
                Manifest m = NavigationManifest.CreateNavigationMetricsFromSavedDataManifest(config.ConfigurationData, config.EnvironmentConfiguration, recordingDir, config.GroundTruthSource, true);
               
                // each navigation should be less than 8 minutes, and there are 15 waypoints per run
                MetricsTestRunner.Run(m, config, TimeSpan.FromMinutes(15 * 8), true);
            }
        }
    }
}
